#!/home/work/rewrite/usr/bin/python
import sys
import pynlp

if len(sys.argv) < 2:
    print 'Usage: %s filepath' % sys.argv[0]
    sys.exit()

filepath = sys.argv[1]
prefix, fileno = filepath.split('-')
fileno = int(fileno)
pynlp.setConf(wdictpath="/home/work/rewrite/dict/worddict/", termnum=2048)

print '''<?xml version="1.0" encoding="utf-8"?>
<sphinx:docset>
<sphinx:schema>
<sphinx:field name="oquery"/> 
<sphinx:field name="query"/>
<sphinx:attr name="freq" type="int"/>
</sphinx:schema>'''

i = 0
for line in file(filepath):
    i += 1
    line = line.replace("&", "&amp;").replace('"', "&quot;").replace("<", "&lt;").replace(">", "&gt;")
    line = unicode(line, 'gb18030', 'ignore').encode('utf-8')

    pieces = line.split(' ||| ')
    if len(pieces) != 2:
        continue
   
    oquery = pieces[0].strip()
    words = pynlp.getParsed(oquery.decode('utf-8').encode('gbk'), pynlp.SCW_BASIC)
    oquery = ''
    for word in words:
        oquery = oquery + word.decode('gbk').encode('utf-8') + ' '
    pieces = pieces[1].split('\t')
    if len(pieces) != 2:
        continue

    query = pieces[0].strip()
    words = pynlp.getParsed(query.decode('utf-8').encode('gbk'), pynlp.SCW_BASIC)
    query = ''
    for word in words:
        query  = query + word.decode('gbk').encode('utf-8') + ' '
    freq = int(pieces[1].strip())
    
    itemno = ((fileno << 32) | i)

    xml  = '<sphinx:document id="%s">\n' % ((fileno << 32) | i)
    xml += '<oquery><![CDATA[%s]]></oquery>\n' % oquery
    xml += '<query><![CDATA[%s]]></query>\n' % query
    xml += '<freq>%s</freq>\n' % freq
    xml += '</sphinx:document>'

    print xml

print '</sphinx:docset>\n'
